{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gzip, os, random\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from utils import Progbar\n",
    "import math\n",
    "%matplotlib inline\n",
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data\n",
    "\n",
    "MNIST can be downloaded [here](http://yann.lecun.com/exdb/mnist/)\n",
    "\n",
    "Training set contains 60, 000 images with labels\n",
    "\n",
    "Testing set contains 10, 000 images with labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28, 1)\n",
      "(60000,)\n"
     ]
    }
   ],
   "source": [
    "def load_data(filename, num_images=60000):\n",
    "    with gzip.open(filename) as bytestream:\n",
    "        bytestream.read(16)\n",
    "        buf = bytestream.read(28 * 28 * num_images)\n",
    "        data = np.frombuffer(buf, dtype=np.uint8).astype(np.float32)\n",
    "        data = data.reshape(num_images, 28, 28, 1)\n",
    "        mean = np.mean(data, axis=0)\n",
    "        data -= mean\n",
    "    return data\n",
    "\n",
    "def load_label(filename, num_images=60000):\n",
    "    with gzip.open(filename) as bytestream:\n",
    "        bytestream.read(8)\n",
    "        buf = bytestream.read(num_images)\n",
    "        labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64)\n",
    "    return labels\n",
    "\n",
    "data = load_data(os.path.join('data', 'train-images-idx3-ubyte.gz'))\n",
    "print(data.shape)\n",
    "labels = load_label(filename = os.path.join('data', 'train-labels-idx1-ubyte.gz'))\n",
    "print(labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 28, 28, 1)\n",
      "(10000,)\n"
     ]
    }
   ],
   "source": [
    "test_data = load_data(os.path.join('data', 't10k-images-idx3-ubyte.gz'), num_images=10000)\n",
    "print(test_data.shape)\n",
    "test_labels = load_label(os.path.join('data', 't10k-labels-idx1-ubyte.gz'), num_images=10000)\n",
    "print(test_labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Take a look at images in training and testing sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x1ef4bfbb518>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAEICAYAAACQ6CLfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEqdJREFUeJzt3W+MXOV1BvDn2Zn1etdrOzY2jmNcE5DT4EaNqbZupdCK\nKi0FkgiiqghLRK6E4lRJaKPmQxCJBB/6AVUFitSWyikWJk2haQFhEGoEqCoKrSIWZLCBEsAxxM5i\nQxwb2+v1zp/TD3OJFrz3vMPembmze56fZHl3zt6Z1+N99s7Oue/70swgIvEMlD0AESmHwi8SlMIv\nEpTCLxKUwi8SlMIvEpTCLxKUwi9nIfl1kuMkz5C8p+zxSHdUyx6A9KWfA/hrAH8MYLjksUiXKPxy\nFjN7EABIjgE4r+ThSJfoZb9IUAq/SFAKv0hQCr9IUHrDT85CsorW90YFQIXkYgB1M6uXOzLpJJ35\nZTbfAXAawI0Arss+/k6pI5KOoxbzEIlJZ36RoBR+kaAUfpGgFH6RoHra6qsOL7HB5St7+ZAiodSO\nH0X99Cm287WFwk/ycgB3otUP/mczu9X7+sHlK3HhdX9V5CFFxPH6v9ze9tfO+WU/yQqAfwBwBYBN\nALaS3DTX+xOR3iryO/8WAK+Z2X4zmwZwP4CrOjMsEem2IuFfB+BnMz4/mN32PiS3Z6vCjDcmTxV4\nOBHppK6/229mO8xszMzGKiNLuv1wItKmIuE/BGD9jM/Py24TkXmgSPifAbCR5MdJLgJwLYDdnRmW\niHTbnFt9ZlYn+XUAP0Sr1bfTzF7s2MhEpKsK9fnN7DEAj3VoLCLSQ7q8VyQohV8kKIVfJCiFXyQo\nhV8kKIVfJCgt3d0HWOYaqkUfu5tjb2tWeneOt6KPPQ/ozC8SlMIvEpTCLxKUwi8SlMIvEpTCLxKU\nWn0dULhVlzo+UXcfP3VsM1VP3EGRf3uinWYD/hdY4tTl1hOPner0LYRWoM78IkEp/CJBKfwiQSn8\nIkEp/CJBKfwiQSn8IkGpz9+m7vbSE/W6/wAD9fxaZdq/74Gaf9+VRJ0N//69hnmz6jfLG4P+YzcW\n+cc3Fzm1wcSxFbcMpvr8qWsY+uA6AZ35RYJS+EWCUvhFglL4RYJS+EWCUvhFglL4RYJSnz+TnJPv\n9OJTffpUL32g5h9fnfKPHzydXx886Tfiqyf9B69M+nXW/Ps3pyFui/1vv1e+POzW//uKO9z6r1VH\nc2sX3v/n7rHLX01cB5C4RsFS1wl4x/boGoBC4Sd5AMAJAA0AdTMb68SgRKT7OnHm/wMze6cD9yMi\nPaTf+UWCKhp+A/AEyWdJbp/tC0huJzlOcrwxeargw4lIpxR92X+JmR0ieS6Ax0n+n5k9NfMLzGwH\ngB0AMPzR9WXuSiciMxQ685vZoezvIwAeArClE4MSke6bc/hJLiG59L2PAVwGYF+nBiYi3VXkZf8a\nAA+x1cetAvhXM/vPjoyqC4r08QG/l5/q41en/PuuOn16AFiU6NUPHs/vxVeP+Q8+cOyEW7dTk359\nOrVgQP755e2tn3IP3brlabf+zNTH3PrU0ERurTrpN9MHEv+sZC8+NZ8/cR1AL8w5/Ga2H8CnOzgW\nEekhtfpEglL4RYJS+EWCUvhFglL4RYJaMFN6k628IttcAxhwls9OLY+dmpK76ITfylt0zH+A6i/y\nL5vmcb+V13w3UT992q3DUk9sfs+rNur3w47VR9z6/unVbv0jlfznpTLlP3ZqyXKrJpYVr6R6ffml\nXm0PrjO/SFAKv0hQCr9IUAq/SFAKv0hQCr9IUAq/SFALps+fVHQbbacVn15627/zwVPOHtsAqsf9\nabk8mT/t1qYS84kTe00PjPi9dm/KLgCc/r1P5tbWfu5N99haYp/sxyb8KcG7//EPc2vnJPYWrw2X\nt8d2r5bu1plfJCiFXyQohV8kKIVfJCiFXyQohV8kKIVfJKh51edPztkvosB1AN5cfwAYmE7UT/t9\nfpxJLBjQyO9Zc/Fi91AuX+TWbdD/Fjn+W2vc+tobXsutfWz4uHts0/xz0y8fWefW1xzKX6vgzGr/\neakNF9hju516H9CZXyQohV8kKIVfJCiFXyQohV8kKIVfJCiFXySoedXnL5W3znpiLYDUdQCs+XfA\nZuIihMVDuSUb8vv4zZH8YwEAVf/8MPF5/xqEP1mxP7d2tL7EPfbfX7nYrV/4wyNuvb5qNLfWrPqN\neBtI1ItO9++D6wCSZ36SO0keIblvxm0rST5O8tXs7xXdHaaIdFo7L/vvAXD5B267EcCTZrYRwJPZ\n5yIyjyTDb2ZPATj6gZuvArAr+3gXgKs7PC4R6bK5vuG3xswmso/fApB7gTfJ7STHSY43JvP3ThOR\n3ir8br+ZGZy3w8xsh5mNmdlYZcR/g0dEemeu4T9Mci0AZH/7b7uKSN+Za/h3A9iWfbwNwMOdGY6I\n9Eqyz0/yPgCXAlhF8iCAmwHcCuAHJK8H8AaAa7o5yPd4vdOuzvVPSPXhmerzO/PxgXTPGYP5vfzm\nqN/Hbw753wK1Zf51At/+7Ufc+kTtI7m1/adWuceu+o9ht26D/jUGzUX557Zknz8xnb8f+vRFJcNv\nZltzSp/t8FhEpId0ea9IUAq/SFAKv0hQCr9IUAq/SFBxpvQWbM1403aTU3pTU3YbBfuUFedneGIL\n7alz/VbgyF8ccusHp1e69cPTy3JrLzxykXvshv99w603Vy136w2n1WeVYlN2i+rVNtwenflFglL4\nRYJS+EWCUvhFglL4RYJS+EWCUvhFgorT508pskV3ok+fXHrbUg8+96Ywz9Tc+tFfX+rWf3PpO279\n1VPnuvWnn/9Ebm3TPa+7x6Liz6tN9uqdqdCF++yp/7LUpRtOvVfXAOjMLxKUwi8SlMIvEpTCLxKU\nwi8SlMIvEpTCLxLUgunzp3qjqb4rE712dz6/v/J28T5+qu7c/9u/42+g/MnP/8StH5zMX3obAJ5/\ncYNbv+jmn+bWGu++6x5bWe0v7Z1aq8CT/H5IrNFQ5lLxnaIzv0hQCr9IUAq/SFAKv0hQCr9IUAq/\nSFAKv0hQC6bPX2T+NNBOX9ebgJ147ESf3qr+z2Am9os+szZ/Tv5lX33aPfbN0/66+0fr/hbdiw/7\n30KNXxzNrVVGl7jHYrDYt6e3jkJyW/V+WFi/y5JnfpI7SR4huW/GbbeQPERyT/bnyu4OU0Q6rZ2X\n/fcAuHyW2+8ws83Zn8c6OywR6bZk+M3sKQD5r91EZF4q8obfDSRfyH4tyL2AnOR2kuMkxxuTpwo8\nnIh00lzDfxeACwBsBjAB4La8LzSzHWY2ZmZjlZHEGzwi0jNzCr+ZHTazhpk1AXwXwJbODktEum1O\n4Se5dsanXwSwL+9rRaQ/JRupJO8DcCmAVSQPArgZwKUkN6PV4T4A4CtdHGNvFLkOINESbg4m+vhD\n/n8DE/PW91+bX79kwF9sYLjir+s/Up126+c/4L8XTKeXz5Fh91hL9PmLruFQ5L4XwmUAyfCb2dZZ\nbr67C2MRkR7S5b0iQSn8IkEp/CJBKfwiQSn8IkEtmCm9Kemlu+d+35b4Edqs+n2hVKvv+EWL3foX\nLh7Pra2qnnSPbQz5g3/i+U1u/aK39rt1DDtjH/KnC6PSvXNTslVXtD4P6MwvEpTCLxKUwi8SlMIv\nEpTCLxKUwi8SlMIvElScPn9iaW6k6kWuA6gklu5O1Ld961G3vrqav9X1ZHPIPfZ7B/x1WC667Zdu\nHRV/WXF6y5Ynls9OGkg8r07dq7Xq/kNbajn2eXAdgM78IkEp/CJBKfwiQSn8IkEp/CJBKfwiQSn8\nIkEtnD5/wZZxcr6/cx0A/dWxMVBPbAdd9y8y+Gj1uFs/0cyfM3+i4S+PXX90lVvHsdf9+rS/9Lc5\nc/JZTXz7pXrpifn+3joKiV3Pk31+zecXkXlL4RcJSuEXCUrhFwlK4RcJSuEXCUrhFwmqnS261wO4\nF8AatLrpO8zsTpIrAfwbgPPR2qb7GjNLTP7uX0XmXzMxL31g2u/jH/iqf/9v15e69aP10dza8USf\nf83/HHPrdsJf9z+FQ856Aqk+/dCgW68P+836xuL8/9TGIv8/vJlIxkK4DqCdM38dwDfNbBOA3wXw\nNZKbANwI4Ekz2wjgyexzEZknkuE3swkzey77+ASAlwGsA3AVgF3Zl+0CcHW3BikinfehfucneT6A\niwH8GMAaM5vISm+h9WuBiMwTbYef5CiABwB8w8zet2icmRlyrq4nuZ3kOMnxxuSpQoMVkc5pK/wk\nB9EK/vfN7MHs5sMk12b1tQCOzHasme0wszEzG6uMLOnEmEWkA5LhZ2v51bsBvGxmt88o7QawLft4\nG4CHOz88EemWdqb0fgbAlwDsJbknu+0mALcC+AHJ6wG8AeCa7gyxTYnWSqqVl5zi6dRTy0Af2+hv\nsX3FJ/K32AaAw7Xlbv3g1Irc2tOPfto9dsP+vW69OXXGrQ8s9pcGp1NvLvVfCdaX+/ddW+p/+9aG\n889tjcTu4JbYVr1oq68flvZOht/MfoT8f8pnOzscEekVXeEnEpTCLxKUwi8SlMIvEpTCLxKUwi8S\n1MJZujsh1Zf1lnkGgLrTch4Y9o+dWunXx0Z/6tbfnPaX137l2Lm5tQ1/5/fx0fDXHa+M+r14LvOn\nGzdX5tdrK0fcY8+s8Kf0To/6/6l1ZzZzc7C7ffz5QGd+kaAUfpGgFH6RoBR+kaAUfpGgFH6RoBR+\nkaAWTJ8/NT+aiR9zlngmGs6U/OnEz9DaMn9p798Y+rlbXzIw7dZfGl2bWzu++hz32NRaBBjyJ77X\nlvlrFdSW5ffqp5f5iyjUlvjPa8Of7u/28puJ9RuSffx5MF8/RWd+kaAUfpGgFH6RoBR+kaAUfpGg\nFH6RoBR+kaAWTJ+/qNS6/Q3mN25Tc7+riV3Knp06361ft/SAW6+t2pNbu2vTn/oPnpCa957a6rox\n5G2TnXrsRL3I2voB+vgpOvOLBKXwiwSl8IsEpfCLBKXwiwSl8IsEpfCLBJXs85NcD+BeAGsAGIAd\nZnYnyVsAfBnA29mX3mRmj3VroEUV7ct66wGk+tHVSb9+199f7db/qeGvB8CmU8uf6t+W1POWusbB\nXS8geWyxepG19RdCHz+lnYt86gC+aWbPkVwK4FmSj2e1O8zsb7s3PBHplmT4zWwCwET28QmSLwNY\n1+2BiUh3fajf+UmeD+BiAD/ObrqB5Askd5JckXPMdpLjJMcbk4nrXEWkZ9oOP8lRAA8A+IaZvQvg\nLgAXANiM1iuD22Y7zsx2mNmYmY1VRvx930Skd9oKP8lBtIL/fTN7EADM7LCZNcysCeC7ALZ0b5gi\n0mnJ8JMkgLsBvGxmt8+4feb7yF8EsK/zwxORbmnn3f7PAPgSgL0k35s7ehOArSQ3o9X+OwDgK10Z\n4XyQaoclpgunWlasFOlZzf3Q1oMn7r5IS6zgtNoiIrTyUtp5t/9HmP2/oW97+iKSpiv8RIJS+EWC\nUvhFglL4RYJS+EWCUvhFgtLS3W3qaj879dhFHrpon78g9dP7l878IkEp/CJBKfwiQSn8IkEp/CJB\nKfwiQSn8IkHRrHeNYJJvA3hjxk2rALzTswF8OP06tn4dF6CxzVUnx7bBzFa384U9Df9ZD06Om9lY\naQNw9OvY+nVcgMY2V2WNTS/7RYJS+EWCKjv8O0p+fE+/jq1fxwVobHNVythK/Z1fRMpT9plfREqi\n8IsEVUr4SV5O8hWSr5G8sYwx5CF5gORekntIjpc8lp0kj5DcN+O2lSQfJ/lq9veseySWNLZbSB7K\nnrs9JK8saWzrSf4XyZdIvkjyL7PbS33unHGV8rz1/Hd+khUAPwHwRwAOAngGwFYze6mnA8lB8gCA\nMTMr/YIQkr8P4CSAe83sU9ltfwPgqJndmv3gXGFm3+qTsd0C4GTZ27Znu0mtnbmtPICrAfwZSnzu\nnHFdgxKetzLO/FsAvGZm+81sGsD9AK4qYRx9z8yeAnD0AzdfBWBX9vEutL55ei5nbH3BzCbM7Lns\n4xMA3ttWvtTnzhlXKcoI/zoAP5vx+UGU+ATMwgA8QfJZktvLHsws1pjZRPbxWwDWlDmYWSS3be+l\nD2wr3zfP3Vy2u+80veF3tkvMbDOAKwB8LXt525es9TtbP/Vq29q2vVdm2Vb+V8p87ua63X2nlRH+\nQwDWz/j8vOy2vmBmh7K/jwB4CP239fjh93ZIzv4+UvJ4fqWftm2fbVt59MFz10/b3ZcR/mcAbCT5\ncZKLAFwLYHcJ4zgLySXZGzEguQTAZei/rcd3A9iWfbwNwMMljuV9+mXb9rxt5VHyc9d3292bWc//\nALgSrXf8Xwfw7TLGkDOuCwA8n/15seyxAbgPrZeBNbTeG7kewDkAngTwKoAnAKzso7F9D8BeAC+g\nFbS1JY3tErRe0r8AYE/258qynztnXKU8b7q8VyQoveEnEpTCLxKUwi8SlMIvEpTCLxKUwi8SlMIv\nEtT/A/xAhokZQJEEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1ef4bef8f28>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "img = data[3].reshape(28,28)\n",
    "plt.imshow(img)\n",
    "plt.title(labels[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x1ef4c1a1438>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAEICAYAAACQ6CLfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAE4NJREFUeJzt3X2MXNV5BvDnmdlv2wteDLaxlxiKEXUJMe0WtQopUBQC\nViNDpFLcCjktqoOaRI1C2yCqKKhqFVQ1UNRUqE6xMJQQUiUU2lAaviSKKJSFGGwgxg7YtY2xwd+7\nXu/uzLz9Yy/RBva+Z9h7Z+7snucnWbs779y5Z2f38Z2d9557aGYQkfiUih6AiBRD4ReJlMIvEimF\nXyRSCr9IpBR+kUgp/CKRUvhlSiR/meSTJI+Q3E7y6qLHJPlS+OVDSLYBeAjAfwDoA7AOwL+QPKfQ\ngUmuqDP85INIngfgOQDzLPkFIfljAM+b2dcLHZzkRkd+qRcBnFf0ICQ/Cr9MZSuA/QD+nGQ7ycsB\nXAygp9hhSZ70sl+mRPJ8AP+AiaP9IIB3AYya2fWFDkxyo/BLXUg+C2Cjmf1T0WORfOhlv0yJ5Pkk\nu0j2kPwzAIsB3F3wsCRHCr+kuQ7AXkz87X8ZgE+b2WixQ5I86WW/SKR05BeJlMIvEimFXyRSCr9I\npNqaurPuOdbe29fMXYpEZfzoQVRGhlnPfTOFn+QVAO4AUAbwz2Z2q3f/9t4+nP37X82ySxFxbP/u\nbXXfd9ov+0mWAfwjgCsBrACwhuSK6T6eiDRXlr/5LwSw3czeNLMxAN8DsDqfYYlIo2UJ/xIAuyZ9\nvTu57ReQXEdykORgdWQ4w+5EJE8Nf7ffzNab2YCZDZS75zR6dyJSpyzh3wOgf9LXS5PbRGQGyBL+\nFwAsJ3kmyQ4A1wJ4OJ9hiUijTbvVZ2YVkl8C8F+YaPVtMLNXcxuZiDRUpj6/mT0C4JGcxiIiTaTT\ne0UipfCLRErhF4mUwi8SKYVfJFIKv0ikmjqfX6bGrNdQzbB95n1nYHXNOndk2D7zvmcBHflFIqXw\ni0RK4ReJlMIvEimFXyRSCr9IpNTqy0GjW3XBx68529b8jYOPnXFsXkvNQoeeQDvOSoE7eOWMh73Z\n0CrUkV8kUgq/SKQUfpFIKfwikVL4RSKl8ItESuEXiZT6/HVy+9mhXrjThwcAVv0HKFX87ctjzrbj\n/mOXx7LVQ9+7ldNrtXa/WV7tCNX9fdfanVrgN9/KGc4hqKPeCucJ6MgvEimFXyRSCr9IpBR+kUgp\n/CKRUvhFIqXwi0Qqmj5/1jn3bq8+0McvB3rtJadPDwDtI/727cPV9NqQf5JAecjfeel4YHCV9H0D\nANrSG/21Lr9RX+316+Nz/V/fsbnp+650h84h8J/zWpu/vXd+A+CfBtCscwAyhZ/kDgDHAFQBVMxs\nII9BiUjj5XHkv9TM3svhcUSkifQ3v0iksobfADxO8kWS66a6A8l1JAdJDlZHhjPuTkTykvVl/0Vm\ntofkaQAeI/lTM3t68h3MbD2A9QDQvbC/wJXhRGSyTEd+M9uTfNwP4EEAF+YxKBFpvGmHn+QckvPe\n/xzA5QC25DUwEWmsLC/7FwJ4kOT7j/NdM3s0l1EVIDTn3uvlh/r45RP+Q7cf93feccTv1XccSe/F\nlw4N+Ts/4tdt2H+fprToNLe+43cXpdZGzzvubrv0Xv/Xs/Nd/4lltcup+o14Y6CPXwpdxCHbeQDN\nMO3wm9mbAD6R41hEpInU6hOJlMIvEimFXyRSCr9IpBR+kUjNmim9WZeaDtVLlfQ7hKfkZmzlHfJb\nWuUDx1JrdviIuy0W+626XX+03K13f8qf09XfszO11tPmP3GHvtrj1ke+vdDf/pz0X+/SuLspug74\nvxAWSE41dFh1Hj40ozevKb868otESuEXiZTCLxIphV8kUgq/SKQUfpFIKfwikZo1ff6Q0HkArPl3\noHOF6tCU3rZAn7/9qN/vLh8MTLsdSp92Wztzqbvt4b8Zdesrere69VLgia1laEr3zz3k1jtv9s8x\nuOzk11Jrzw39krvtc7cHLkTdCmtsZ6Qjv0ikFH6RSCn8IpFS+EUipfCLRErhF4mUwi8SqRnV58+6\nzLYrdB6AN/86sEp1+YTf5y8P+31+6/EuQQ3svHZJau3Sz73objunze/zdwUmvu8eme/W57WnX4tg\nuNLpbhtyetdht15yrse+6qSX3W3/h7/u7zzU5p8BpwHoyC8SKYVfJFIKv0ikFH6RSCn8IpFS+EUi\npfCLRGpG9fkbKnixdGfTwLUASpXA+t/j/nX7t95wqlu/6uLnUmtLOv1eeGegj//tBz7r1ue8HbgO\nwuoDqbXfXPSWu23N/GNTJ/3nbffYKam1v7r3D9xt51pozfaMjXxn82ZdKiB45Ce5geR+klsm3dZH\n8jGS25KP/pkeItJy6nnZfzeAKz5w200AnjCz5QCeSL4WkRkkGH4zexrAwQ/cvBrAxuTzjQCuynlc\nItJg033Db6GZ7U0+fwdA6qJpJNeRHCQ5WB1Jv9aciDRX5nf7zczgvB1mZuvNbMDMBsrdc7LuTkRy\nMt3w7yO5GACSj/vzG5KINMN0w/8wgLXJ52sBPJTPcESkWYJ9fpL3A7gEwAKSuwF8A8CtAL5P8noA\nOwFc08hBvs/rf4bm+od6p1laq6XAfH5U/cFt/XqvW//sikG33tuWPmf+mYP+9el3bzjbrY+f5/e7\nz7juZ259Re87qbWD4/6fgW2BCyX0lP1rETy671dSa/O3+Y89Nsc/LgZ78TNgPn8w/Ga2JqV0Wc5j\nEZEm0um9IpFS+EUipfCLRErhF4mUwi8SKU3prZNzFWiUAkt07/idbrf+h594yq0fr3a49RcPn5Fa\ne+tHZ7nb3vAX/+7WR2vtbn3v2El+/UR6faTqP3Zfx3G3/r+Hz3Trhzf0p9baSqFrtfvloEZeZj4n\nOvKLRErhF4mUwi8SKYVfJFIKv0ikFH6RSCn8IpGaNX3+4JTdUN81UC8503IDV5jG713xjP/YgZ13\nlvxLVF9yyhuptVWf3+xuu2/c79M/f3CZW9/zb3796Ir0S4N//Nxd7rahPv8L/32uW+/fn77vsd6y\nu22WJdtnCh35RSKl8ItESuEXiZTCLxIphV8kUgq/SKQUfpFIzZo+f1befH0Abt+3VPGbvi8dSp9X\nDgCXnrrV33dgbO+Nz02tHRhPXUkNAPDMgxe49Y/d+apbX9r3tls/8Jn0X7Fa4OSMJ3cud+vLfpR+\nyXIAqHY7vfxgHz/jteBnAB35RSKl8ItESuEXiZTCLxIphV8kUgq/SKQUfpFIqc+fCM3PduuBbU98\n83S3fs/5y9x6YCVqzN+WPm+9Z/Med9v+/f7y36HTH966cYVbP6Nzd2pttOr/+i25w7+uf/n4mFuv\ndXSl1oLndUQgeOQnuYHkfpJbJt12C8k9JDcl/1Y1dpgikrd6XvbfDeCKKW6/3cxWJv8eyXdYItJo\nwfCb2dMADjZhLCLSRFne8PsyyVeSPwvmp92J5DqSgyQHqyPDGXYnInmabvjvBHAWgJUA9gL4Vtod\nzWy9mQ2Y2UC5e840dycieZtW+M1sn5lVzawG4DsALsx3WCLSaNMKP8nFk768GsCWtPuKSGsK9vlJ\n3g/gEgALSO4G8A0Al5BciYkO9w4AX2jgGOuSqU8PhNdT9+qBqd3lkapbX/LUEbfOit+U5rjz+G3+\n9elL3em9cAA4fKXfx1/+qR1uvaucfg7Cy8/68/XP2eWfo1CdP8+ts5b+QwvO1w/9UGf+dP5w+M1s\nzRQ339WAsYhIE+n0XpFIKfwikVL4RSKl8ItESuEXiVQ8U3qztvq8TUt+38fa/Hqtw/8xlEqB+ad0\nHr/a6W5qZ/jTjYfW+G3Iszv9U7a3HT41tXbO37/lbosuf+ysBZ4X52dq3nNWV93f9UxoBerILxIp\nhV8kUgq/SKQUfpFIKfwikVL4RSKl8ItESn3+RHBKsNdSDq7mHGj6hs4TKAe2rzn1Tv9HvO+v/cH/\n9pLtbv3YuD8luPKvp6XWbMxfmpwd/qW7EZiW651/YYHDXtb6TDALvgURmQ6FXyRSCr9IpBR+kUgp\n/CKRUvhFIqXwi0Rq9vT5g338UDM+tL1Tcy4RDQClauDS26H6qH/pb+/S3aOL/FWSfm3hT916qI//\n5Ev+pb3PfeCV9GJHh7ute50CANbuX5a81p6+fS1wjQXzHzo4Xz84378F6MgvEimFXyRSCr9IpBR+\nkUgp/CKRUvhFIqXwi0SqniW6+wHcA2AhJrrh683sDpJ9AB4AsAwTy3RfY2aHGjfUbIJz6jNcuN+d\n6w+AlcB5ACPpy1gDAEfG3Hr15J7Umn3tPXfb0Zrf0P7JO0vd+opvvu3Wq2Pp3xt70scNANbtX7e/\nOtc/T6DSk/69VTsDaykEkhFaq2EmqOfIXwFwo5mtAPAbAL5IcgWAmwA8YWbLATyRfC0iM0Qw/Ga2\n18xeSj4/BuB1AEsArAawMbnbRgBXNWqQIpK/j/Q3P8llAC4A8DyAhWa2Nym9g4k/C0Rkhqg7/CTn\nAvgBgK+Y2dHJNTMzpPzRTHIdyUGSg9URf103EWmeusJPsh0Twb/PzH6Y3LyP5OKkvhjA/qm2NbP1\nZjZgZgPlbn+SiYg0TzD8JAngLgCvm9ltk0oPA1ibfL4WwEP5D09EGqWeKb2fBHAdgM0kNyW33Qzg\nVgDfJ3k9gJ0ArmnMEHMSmoIZmMLpdcSyXsaZlcCU3pFRtz58/oLU2qKuvak1AHjvxFy33vHoSW69\nsus1t17u7U0vnuTvuzLfbwWO9fqX9h7vSf+hVwOziUNTfoNLcM+AKb/B8JvZM0j/Vi7Ldzgi0iw6\nw08kUgq/SKQUfpFIKfwikVL4RSKl8ItEavZcujtj3zU0hdObAlrp8f8PLY/6JxGUOvydH//V0936\nyV/6v/THDqw9Xq35Y1/wkyG3Xj6lz62j7+TUUuU05xwAAKN9fjN+tNcfe6XbuXR3YPXv0GFRS3SL\nyIyl8ItESuEXiZTCLxIphV8kUgq/SKQUfpFIzZo+f3B+dOi/ucD87Wpner98LNj09Z/m8bn+FY7G\nbzjg1i9e8EZq7dC4/9hv2Glu/cB8//LZXf2L3Pr4/PQlvkdPDszHn+P/TLw+PgBUnSW6G93Hb4X5\n+iE68otESuEXiZTCLxIphV8kUgq/SKQUfpFIKfwikZo1ff6QrOcB1JyesZX8OfO1dv/Bhy7358wP\nfvw+t364VkmtPXtiibvtf951kVs/qc1fPnzorHluveJcByG0THbma+s7T3sMffwQHflFIqXwi0RK\n4ReJlMIvEimFXyRSCr9IpBR+kUgF+/wk+wHcA2AhAAOw3szuIHkLgD8G8G5y15vN7JFGDbTRspwH\nYPQ3DvWUe7rG3PrcUvqceAD4zJbPpdaOPu7Pt+8a9s9RGF7o/4qEvjdzliwIbxt4XjOu1ZDpsWeB\nek7yqQC40cxeIjkPwIskH0tqt5vZ3zVueCLSKMHwm9leAHuTz4+RfB2Af9qYiLS8j/Q3P8llAC4A\n8Hxy05dJvkJyA8n5KdusIzlIcrA6MpxpsCKSn7rDT3IugB8A+IqZHQVwJ4CzAKzExCuDb021nZmt\nN7MBMxsod/vXkxOR5qkr/CTbMRH8+8zshwBgZvvMrGpmNQDfAXBh44YpInkLhp8kAdwF4HUzu23S\n7Ysn3e1qAFvyH56INEo97/Z/EsB1ADaT3JTcdjOANSRXYqL9twPAFxoywpkg0BYKtrQeO8WtX/Dj\nPwk8QHqpze/kBS9/HZSh3dbIVl1IDK28kHre7X8GU/8YZmxPX0R0hp9ItBR+kUgp/CKRUvhFIqXw\ni0RK4ReJVDSX7s4qU184Y0850KpvKDZw5+q1F0tHfpFIKfwikVL4RSKl8ItESuEXiZTCLxIphV8k\nUjRrXheZ5LsAdk66aQGA95o2gI+mVcfWquMCNLbpynNsHzOzU+u5Y1PD/6Gdk4NmNlDYABytOrZW\nHRegsU1XUWPTy36RSCn8IpEqOvzrC96/p1XH1qrjAjS26SpkbIX+zS8ixSn6yC8iBVH4RSJVSPhJ\nXkFyK8ntJG8qYgxpSO4guZnkJpKDBY9lA8n9JLdMuq2P5GMktyUfp1wjsaCx3UJyT/LcbSK5qqCx\n9ZN8iuRrJF8l+afJ7YU+d864Cnnemv43P8kygDcAfBrAbgAvAFhjZq81dSApSO4AMGBmhZ8QQvK3\nAAwBuMfMzktu+1sAB83s1uQ/zvlm9rUWGdstAIaKXrY9WU1q8eRl5QFcBeDzKPC5c8Z1DQp43oo4\n8l8IYLuZvWlmYwC+B2B1AeNoeWb2NICDH7h5NYCNyecbMfHL03QpY2sJZrbXzF5KPj8G4P1l5Qt9\n7pxxFaKI8C8BsGvS17tR4BMwBQPwOMkXSa4rejBTWGhme5PP3wGwsMjBTCG4bHszfWBZ+ZZ57qaz\n3H3e9Ibfh11kZisBXAngi8nL25ZkE3+ztVKvtq5l25tlimXlf67I5266y93nrYjw7wHQP+nrpclt\nLcHM9iQf9wN4EK239Pi+91dITj7uL3g8P9dKy7ZPtaw8WuC5a6Xl7osI/wsAlpM8k2QHgGsBPFzA\nOD6E5JzkjRiQnAPgcrTe0uMPA1ibfL4WwEMFjuUXtMqy7WnLyqPg567llrs3s6b/A7AKE+/4/wzA\nXxYxhpRxnQXg5eTfq0WPDcD9mHgZOI6J90auB3AKgCcAbAPwOIC+FhrbvQA2A3gFE0FbXNDYLsLE\nS/pXAGxK/q0q+rlzxlXI86bTe0UipTf8RCKl8ItESuEXiZTCLxIphV8kUgq/SKQUfpFI/T+nutlK\neiZpugAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1ef4bef8390>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "img = test_data[20].reshape(28,28)\n",
    "plt.imshow(img)\n",
    "plt.title(test_labels[20])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Randomize the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_indicis(data_size, batch_size, shuffle=True):\n",
    "    indicis = np.arange(data_size)\n",
    "    if shuffle:\n",
    "        np.random.shuffle(indicis)\n",
    "    for start_idx in range(0, data_size, batch_size):\n",
    "        yield indicis[start_idx : start_idx + batch_size]\n",
    "        \n",
    "def load_mnist_data(data, labels, batch_size, shuffle=True):\n",
    "    data_size = data.shape[0]\n",
    "    for idx in get_indicis(data_size, batch_size, shuffle):\n",
    "        yield data[idx], labels[idx]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define MNIST training model\n",
    "\n",
    "A simple model:\n",
    "CONV3 - MAX POOL - CONV3 - MAX POOL - FC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class MNIST_model(object):\n",
    "    def __init__(self, num_classes, learning_rate=1e-3):\n",
    "        tf.reset_default_graph()\n",
    "        self.num_classes = num_classes\n",
    "        self.lr = learning_rate\n",
    "        self.input_placeholder = None\n",
    "        self.label_placeholder = None\n",
    "        self.training_placeholder = None\n",
    "        self.hist = []\n",
    "        self.build()\n",
    "        \n",
    "    def build(self):\n",
    "        self.add_placeholders()\n",
    "        self.pred = self.add_prediction_op()\n",
    "        self.loss, self.corr, self.acc = self.add_loss_op(self.pred)\n",
    "        self.train_op = self.add_train_op(self.loss)\n",
    "        \n",
    "    def add_placeholders(self):\n",
    "        self.input_placeholder = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))\n",
    "        self.label_placeholder = tf.placeholder(tf.int64, shape=(None))\n",
    "        self.training_placeholder = tf.placeholder(tf.bool)\n",
    "        \n",
    "    def add_loss_op(self, pred):\n",
    "        y = self.label_placeholder\n",
    "        cross_entropy = tf.nn.softmax_cross_entropy_with_logits(\n",
    "            labels=tf.one_hot(y, self.num_classes),\n",
    "            logits = pred\n",
    "        )\n",
    "        loss = tf.reduce_mean(cross_entropy)\n",
    "        \n",
    "        correct_prediction = tf.equal(tf.argmax(pred, 1), y)\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "        \n",
    "        return loss, correct_prediction, accuracy\n",
    "        \n",
    "    def add_train_op(self, loss):\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate=self.lr)\n",
    "        \n",
    "        update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "        with tf.control_dependencies(update_ops):\n",
    "            train_op = optimizer.minimize(loss)\n",
    "            \n",
    "        return train_op\n",
    "    \n",
    "    def create_feed_dict(self, inputs_batch, labels_batch=None, is_training=True):\n",
    "        feed_dict = {\n",
    "            self.input_placeholder : inputs_batch,\n",
    "            self.training_placeholder : is_training,\n",
    "        }\n",
    "        if labels_batch is not None:\n",
    "            feed_dict[self.label_placeholder] = labels_batch\n",
    "        return feed_dict\n",
    "    \n",
    "    def train_on_batch(self, sess, inputs_batch, labels_batch=None, is_training=True):\n",
    "        feed = self.create_feed_dict(inputs_batch, labels_batch, is_training)\n",
    "        variables = [self.loss, self.corr, self.acc]\n",
    "        if is_training:\n",
    "            variables[-1] = self.train_op \n",
    "        loss, corr, _ = sess.run(variables, feed_dict=feed)\n",
    "        \n",
    "        return loss, corr\n",
    "    \n",
    "    def run_epoch(self, sess, batch_size, training_set, validation_set):\n",
    "        X_tr, Y_tr = training_set\n",
    "        \n",
    "        prog = Progbar(target=int(math.ceil(X_tr.shape[0] / batch_size)))\n",
    "        \n",
    "        for i, (train_x, train_y) in enumerate(load_mnist_data(X_tr, Y_tr, batch_size)):\n",
    "            loss, corr = self.train_on_batch(sess, train_x, train_y, True)\n",
    "            prog.update(i + 1, [('train_loss', loss), ('train_acc', np.sum(corr)/train_x.shape[0])])\n",
    "            \n",
    "        X_val, Y_val = validation_set\n",
    "            \n",
    "        prog = Progbar(target=int(math.ceil(X_val.shape[0] / batch_size)))\n",
    "        \n",
    "        hist_input = []\n",
    "        hist_pred = []\n",
    "        for i, (val_x, val_y) in enumerate(load_mnist_data(X_val, Y_val, batch_size, shuffle=False)):\n",
    "            loss, corr = self.train_on_batch(sess, val_x, val_y, is_training=False)\n",
    "            hist_input += val_x.tolist()\n",
    "            hist_pred += corr.tolist()\n",
    "            prog.update(i + 1, [('val_loss', loss), ('val_acc', np.sum(corr)/val_x.shape[0])])\n",
    "        self.hist = [(a,b) for a, b in zip(hist_input, hist_pred)]\n",
    "            \n",
    "    def fit(self, sess, epoches, batch_size, training_set, validation_set):\n",
    "        for e in range(epoches):\n",
    "            print(\"\\nEpoch {:} / {:}\".format(e + 1, epoches))\n",
    "            self.run_epoch(sess, batch_size, training_set, validation_set)\n",
    "            \n",
    "    def add_prediction_op(self):\n",
    "        '''\n",
    "        Return predictions\n",
    "        '''\n",
    "        is_training = self.training_placeholder\n",
    "        \n",
    "        x = self.input_placeholder\n",
    "        x = tf.contrib.layers.conv2d(\n",
    "            inputs=x,\n",
    "            num_outputs=10,\n",
    "            kernel_size=3,\n",
    "            padding='SAME',\n",
    "            activation_fn=None\n",
    "        ) # 28x28x10\n",
    "        x = tf.contrib.layers.batch_norm(\n",
    "            x,\n",
    "            is_training=is_training\n",
    "        )\n",
    "        x = tf.nn.relu(x)\n",
    "        x = tf.contrib.layers.max_pool2d(\n",
    "            inputs=x,\n",
    "            kernel_size=2,\n",
    "            stride=2,\n",
    "            padding='VALID'\n",
    "        ) # 14x14x10\n",
    "        \n",
    "        x = tf.contrib.layers.conv2d(\n",
    "            inputs=x,\n",
    "            num_outputs=10,\n",
    "            kernel_size=3,\n",
    "            padding='SAME',\n",
    "            activation_fn=None\n",
    "        ) # 14x14x10\n",
    "        x = tf.contrib.layers.batch_norm(\n",
    "            x,\n",
    "            is_training=is_training\n",
    "        )\n",
    "        x = tf.nn.relu(x)\n",
    "        x = tf.contrib.layers.max_pool2d(\n",
    "            inputs=x,\n",
    "            kernel_size=2,\n",
    "            stride=2,\n",
    "            padding='VALID'\n",
    "        ) # 7x7x10\n",
    "        \n",
    "        x = tf.contrib.layers.flatten(x)\n",
    "        \n",
    "        pred = tf.contrib.layers.fully_connected(\n",
    "            x,\n",
    "            self.num_classes,\n",
    "            activation_fn=None\n",
    "        )\n",
    "        return pred\n",
    "        \n",
    "        \n",
    "model = MNIST_model(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 1 / 5\n",
      "938/938 [==============================] - 9s - train_loss: 0.2723 - train_acc: 0.9194     \n",
      "157/157 [==============================] - 3s - val_loss: 0.1091 - val_acc: 0.9671     - ETA: 0s \n",
      "\n",
      "Epoch 2 / 5\n",
      "938/938 [==============================] - 9s - train_loss: 0.0830 - train_acc: 0.9750     \n",
      "157/157 [==============================] - 4s - val_loss: 0.0588 - val_acc: 0.9808     \n",
      "\n",
      "Epoch 3 / 5\n",
      "938/938 [==============================] - 9s - train_loss: 0.0630 - train_acc: 0.9810     \n",
      "157/157 [==============================] - 4s - val_loss: 0.0549 - val_acc: 0.9827     - ETA: 0\n",
      "\n",
      "Epoch 4 / 5\n",
      "938/938 [==============================] - 9s - train_loss: 0.0531 - train_acc: 0.9838     - ETA: 0s - train_loss: 0.0531 - train_acc: 0\n",
      "157/157 [==============================] - 3s - val_loss: 0.0527 - val_acc: 0.9821     \n",
      "\n",
      "Epoch 5 / 5\n",
      "938/938 [==============================] - 9s - train_loss: 0.0463 - train_acc: 0.9861     \n",
      "157/157 [==============================] - 4s - val_loss: 0.0448 - val_acc: 0.9850     \n"
     ]
    }
   ],
   "source": [
    "model.fit(sess, 5, 64, (data, labels), (test_data, test_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Take a look at the predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "incorrect = [x[0] for i,x in enumerate(model.hist) if x[1] == False]\n",
    "correct = [x[0] for i, x in enumerate(model.hist) if x[1] == True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "151 9849\n"
     ]
    }
   ],
   "source": [
    "print(len(incorrect), len(correct))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1f0444bc780>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAETlJREFUeJzt3W+MleWZx/HfJQ4M/wyiMkwUtRr/xmSnZDTG4upa21pT\no0Qh9YVh1ZSadJttaOIa98Wa4Auz2VZ5sTaZKimuXVqTlugLghEjEhNjGP9UUdbVBVQIDBCQjvyV\n8doX89CMOue6x/Occ54zc38/CZmZc517zj0P/DhnzvU8923uLgD5OaXqCQCoBuEHMkX4gUwRfiBT\nhB/IFOEHMkX4gUwRfiBThB/I1KmtfLDOzk6fOXNmKx8SyMrg4KCOHj1qY7lvqfCb2U2SVkiaJOkJ\nd38kuv/MmTO1cOHCMg8JILBmzZox37ful/1mNknSf0r6oaTLJd1pZpfX+/0AtFaZ3/mvkvShu291\n9+OS/iDp1sZMC0CzlQn/2ZI+GfH1juK2LzGzpWbWb2b9R48eLfFwABqp6e/2u3ufu/e6e29nZ2ez\nHw7AGJUJ/05J80Z8fU5xG4BxoEz4N0m6yMy+ZWaTJf1Y0nONmRaAZqu71efuJ8zsnyQ9r+FW30p3\nf7dhMwPQVKX6/O6+VtLaBs0FQAtxei+QKcIPZIrwA5ki/ECmCD+QKcIPZKql1/OjOaJdl8ruyPTF\nF1+UGn/KKfU/v5iN6bL0po2f6HjmBzJF+IFMEX4gU4QfyBThBzJF+IFM0eprgVS7LVVPtdui8WXG\nNqIetdtSbcBUqy41PqqnvncObUKe+YFMEX4gU4QfyBThBzJF+IFMEX4gU4QfyBR9/gZI9dJT9aGh\nobD++eef110/duxYOPb48eOl6qk+/6RJk2rWOjo6wrFTpkwJ65MnT667fuqp8T/91DkE0c8ljY/z\nBHjmBzJF+IFMEX4gU4QfyBThBzJF+IFMEX4gU6X6/Ga2XdKgpCFJJ9y9txGTqkKZ69bL9ulTvfhD\nhw6F9UsuuaRmrbu7Oxx7xx13hPXDhw+H9ZUrV4b1qJ9+4MCBcOybb74Z1mfOnBnWZ8yYUbM2bdq0\ncGzqHIKUMmsNtEojTvL5B3ff14DvA6CFqv/vB0AlyobfJa03s9fNbGkjJgSgNcq+7F/g7jvNbI6k\nF8zsf9x948g7FP8pLJXi38EAtFapZ35331l83CNpjaSrRrlPn7v3untvZ2dnmYcD0EB1h9/MppvZ\nzJOfS/q+pM2NmhiA5irzsr9L0pri0sVTJf23u69ryKwANF3d4Xf3rZL+roFzaaqy689HvfxUHz/V\nK0/VUz3pZcuW1ayl1hIYHBwM60eOHAnrixYtCuup6+YjzzzzTFg/ePBgWN+yZUvdj51Stk8fXe/f\nqrUAaPUBmSL8QKYIP5Apwg9kivADmSL8QKayWbq77DbYJ06cqFlLXZKbapellsdevHhxWB8YGKhZ\n2717dzj28ccfD+t79+4N66njOn/+/Jq1W265JRx71113hfVZs2aF9VWrVtWsvfzyy+HY1LLiqXqq\nFRgdN1p9AJqK8AOZIvxApgg/kCnCD2SK8AOZIvxAprLp86eU2UY7dUlvqs9/8cUXh/Xrr78+rC9f\nvrxmbfXq1eHYVJ8+JdWT3ry59vou77//fjj2scceC+up8yOiZcuj8zak8tuujwc88wOZIvxApgg/\nkCnCD2SK8AOZIvxApgg/kKkJ0+cve71+mfGpnnHqev/U0t1Hjx4N65GpU6eG9SlTpoT11HXpPT09\nYf3RRx+tWUsdl5TUOQYrVqyoWTvjjDNKPXZKam6tumY/wjM/kCnCD2SK8AOZIvxApgg/kCnCD2SK\n8AOZSvb5zWylpB9J2uPuVxS3zZb0R0nnS9ouabG7H2jeNMtL9avLXJ9dZs3/sdRT7rvvvpq1DRs2\nhGNTff7UeQL33HNPWO/s7KxZi9ZIkNLH9emnnw7r0fbgqX8PZesp46XP/ztJN33ltgckvejuF0l6\nsfgawDiSDL+7b5S0/ys33yrp5HYoqyTd1uB5AWiyel+7dLn7ruLz3ZK6GjQfAC1S+g0/Hz4pvuaJ\n8Wa21Mz6zay/zDnqABqr3vAPmFm3JBUf99S6o7v3uXuvu/dGb/4AaK16w/+cpCXF50skPduY6QBo\nlWT4zWy1pFclXWJmO8zsXkmPSPqemX0g6cbiawDjSLLP7+531ih9t8FzKSXVNy27Pn00PtWvTvXx\nU+MnTZoU1ru6ar/fOnfu3HBsai2B6HtL0jXXXBPWo/d5Un38/fu/2mT6snXr1oX1qM/f0dERjk1p\nhz59WZzhB2SK8AOZIvxApgg/kCnCD2SK8AOZmjBLd6eUbc1EbanUFt1lL9lNtfqiltbdd98djo2W\n1pakQ4cOhfXBwcGwHrX6Usdl7dq1YX3fvn1h/cwzz6xZK3vJbtnWcTvgmR/IFOEHMkX4gUwRfiBT\nhB/IFOEHMkX4gUxl0+dPSfVto8tuy27//dprr4X1/v7+sL5gwYKatYULF4Zjd+7cGdZTffwDB+IV\n26Ne/p49NReAkiQ9//zzYT11/kNUT533UWYp9/GCZ34gU4QfyBThBzJF+IFMEX4gU4QfyBThBzKV\nTZ8/1Wsv06sve2136trxvr6+sH7hhRfWrJ111lnh2Ntui/dY/eyzz8J66jyAqNf+0ksvhWM//vjj\nsD5nzpywHv29lO3jT4TzAHjmBzJF+IFMEX4gU4QfyBThBzJF+IFMEX4gU8k+v5mtlPQjSXvc/Yri\ntock/UTS3uJuD7p7vMh6m0v16sv08susuy9J27ZtC+vLli2rWbvuuuvCsT09PWE9tSdBtDa+FP/s\nr7zySjg2dVxSol582TUYJoKxPPP/TtJNo9z+qLv3FH/GdfCBHCXD7+4bJe1vwVwAtFCZ3/l/bmZv\nm9lKMzu9YTMC0BL1hv83ki6Q1CNpl6Rf1bqjmS01s34z64/2bQPQWnWF390H3H3I3b+Q9FtJVwX3\n7XP3Xnfv7ezsrHeeABqsrvCbWfeILxdK2tyY6QBolbG0+lZLul7SmWa2Q9K/SbrezHokuaTtkn7a\nxDkCaIJk+N39zlFufrIJc2mqVF+3TN83dT1+ql/d0dER1lNrzO/YsaNm7YknngjHpq7HT83t4Ycf\nDutXXnllzdrSpUvDscuXLw/rKdGeAak+fuqYp/7Ox4Px/xMAqAvhBzJF+IFMEX4gU4QfyBThBzKV\nzdLdKWUu6U21hVKtvsmTJ4f1MltRp6R+7tQp2alttK+99tqatXPOOScc29XVFdYPHz4c1iOpv7Nm\nj28HPPMDmSL8QKYIP5Apwg9kivADmSL8QKYIP5CpCdPnL7v0dpl6sy8PTV1uHH3/sucYDA0NhfUN\nGzaE9Wju5513Xjj23HPPDevvvfdeWI/Ofyh7GXZq/Hi45Lf9ZwigKQg/kCnCD2SK8AOZIvxApgg/\nkCnCD2RqwvT5y0r16qN+d6oPn+qVR0tMS+ltsqPxZb/38ePHS43ft29fzdqcOXPCsWXXSYiWHU8t\nSZ5aI4Hr+QGMW4QfyBThBzJF+IFMEX4gU4QfyBThBzKV7POb2TxJT0nqkuSS+tx9hZnNlvRHSedL\n2i5psbsfaN5Um6tM3zbV50/12lO99CNHjoT1qNee+t6pdflT9dTPvnXr1pq1uXPnhmM7OzvD+vTp\n08P61KlTa9amTJkSji17Pf94OA9gLM/8JyT90t0vl3S1pJ+Z2eWSHpD0ortfJOnF4msA40Qy/O6+\ny93fKD4flLRF0tmSbpW0qrjbKkm3NWuSABrvG/3Ob2bnS/q2pNckdbn7rqK0W8O/FgAYJ8YcfjOb\nIelPkn7h7n8dWfPhE+NHPTnezJaaWb+Z9ad+fwTQOmMKv5l1aDj4v3f3Pxc3D5hZd1HvlrRntLHu\n3ufuve7em3oDB0DrJMNvw29bPilpi7v/ekTpOUlLis+XSHq28dMD0CxjuaT3O5LukvSOmb1V3Pag\npEckPWNm90r6SNLi5kyxMcounx1dAppqCy1atCisHzx4MKyvW7curG/btq1mbd68eeHYyy67LKyn\n2pQ33nhjWL/66qtr1k477bRw7KxZs8J6anzUCkwtWV621TceJMPv7q9IqpWc7zZ2OgBaZfz/9wWg\nLoQfyBThBzJF+IFMEX4gU4QfyNSEWbq7mX18Kb4E9NJLLw3H3n///WH9008/Deu33357WI+Wx071\ns6dNmxbWU8c1Nf7000+vWZs9e3Y4dv78+WF9YGAgrJe5pLfs0t0T5ZJeABMQ4QcyRfiBTBF+IFOE\nH8gU4QcyRfiBTE2YPn9Kqu+a6utGqxDNmDGjrjmdlOp3p+bW3d1ds3bs2LFwbGrp7dT5D6mfPbom\nP/W9f/CDH4T1jRs3hvXo++fQx0/hmR/IFOEHMkX4gUwRfiBThB/IFOEHMkX4gUzR5y+k+r7RegB7\n9+4Nxy5fvjysp9YDSG1zFvXyo+27JWl4p7XabrjhhrC+fv36sB5dN59ax2DTpk1hPXWeQPR3mkMf\nP4VnfiBThB/IFOEHMkX4gUwRfiBThB/IFOEHMpXs85vZPElPSeqS5JL63H2FmT0k6SeSTja5H3T3\ntc2aaLOV6fum9nL/5JNPwvpHH30U1oeGhuqup67XT3n11VfDepnzJ1LnVqT6+Km9GKK55dDHTxnL\nST4nJP3S3d8ws5mSXjezF4rao+7+H82bHoBmSYbf3XdJ2lV8PmhmWySd3eyJAWiub/Q7v5mdL+nb\nkl4rbvq5mb1tZivNbNR9mcxsqZn1m1l/6jRVAK0z5vCb2QxJf5L0C3f/q6TfSLpAUo+GXxn8arRx\n7t7n7r3u3hutgwegtcYUfjPr0HDwf+/uf5Ykdx9w9yF3/0LSbyVd1bxpAmi0ZPht+G3RJyVtcfdf\nj7h95JKxCyVtbvz0ADTLWN7t/46kuyS9Y2ZvFbc9KOlOM+vRcPtvu6SfNmWG40CZltNYxpfdXjyS\nuqS3bEssNfcyj027rpyxvNv/iqTRjvK47ekD4Aw/IFuEH8gU4QcyRfiBTBF+IFOEH8hUNkt3V6ls\nv7pMrxyohX9VQKYIP5Apwg9kivADmSL8QKYIP5Apwg9kylLXczf0wcz2Shq5TvWZkva1bALfTLvO\nrV3nJTG3ejVybue5+1ljuWNLw/+1Bzfrd/feyiYQaNe5teu8JOZWr6rmxst+IFOEH8hU1eHvq/jx\nI+06t3adl8Tc6lXJ3Cr9nR9Adap+5gdQkUrCb2Y3mdn7ZvahmT1QxRxqMbPtZvaOmb1lZv0Vz2Wl\nme0xs80jbpttZi+Y2QfFx1G3Satobg+Z2c7i2L1lZjdXNLd5ZvaSmb1nZu+a2T8Xt1d67IJ5VXLc\nWv6y38wmSfpfSd+TtEPSJkl3uvt7LZ1IDWa2XVKvu1feEzazv5f0maSn3P2K4rZ/l7Tf3R8p/uM8\n3d3/pU3m9pCkz6reubnYUKZ75M7Skm6T9I+q8NgF81qsCo5bFc/8V0n60N23uvtxSX+QdGsF82h7\n7r5R0v6v3HyrpFXF56s0/I+n5WrMrS24+y53f6P4fFDSyZ2lKz12wbwqUUX4z5b0yYivd6i9tvx2\nSevN7HUzW1r1ZEbRVWybLkm7JXVVOZlRJHdubqWv7CzdNseunh2vG403/L5ugbv3SPqhpJ8VL2/b\nkg//ztZO7Zox7dzcKqPsLP03VR67ene8brQqwr9T0rwRX59T3NYW3H1n8XGPpDVqv92HB05uklp8\n3FPxfP6mnXZuHm1nabXBsWunHa+rCP8mSReZ2bfMbLKkH0t6roJ5fI2ZTS/eiJGZTZf0fbXf7sPP\nSVpSfL5E0rMVzuVL2mXn5lo7S6viY9d2O167e8v/SLpZw+/4/5+kf61iDjXmdYGkvxR/3q16bpJW\na/hl4Ocafm/kXklnSHpR0geS1kua3UZz+y9J70h6W8NB665obgs0/JL+bUlvFX9urvrYBfOq5Lhx\nhh+QKd7wAzJF+IFMEX4gU4QfyBThBzJF+IFMEX4gU4QfyNT/A+CN9AMQNQ+4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1f07547a4a8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n = random.randint(0, len(incorrect))\n",
    "img = np.array(incorrect[n]).reshape(28,28)\n",
    "plt.imshow(img,cmap='gray')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final Thoughts\n",
    "MNIST is quoted in almost all the deep learning books, e.g.[neural network and deep learning](http://neuralnetworksanddeeplearning.com/). Google already has a [tutorial](https://www.tensorflow.org/get_started/mnist/pros) on this using Tensorflow. The network setup is a bit different from the tutorial from Google. (I'm not a fan of dropout). I've added 2 layers of batch normalization between the two conv layers which is supposed to get me to the point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
